{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "['Current_1', 'Current_2', 'Current_3'，'Vibration_1']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import joblib\n",
    "import os\n",
    "from joblib import Parallel, delayed"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.sensor文件整合"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 常量\n",
    "# 只使用新的数据集(train_add),旧的数据集(train_qLua)存在着电流和振动信号名称混淆的问题\n",
    "# old_input_dir = '../raw_data/train_qLua/'\n",
    "new_input_dir = '../raw_data/train_add/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def integrate_sensors(train_no, csv_nos):\n",
    "    '''\n",
    "    描述：\n",
    "        整合所有的sensor文件，并分别输出\n",
    "    '''\n",
    "    if not os.path.exists('./sensors'):\n",
    "        os.mkdir('./sensors')\n",
    "        \n",
    "#     old_input_subdir = old_input_dir + '0%d/Sensor/'%train_no\n",
    "    new_input_subdir = new_input_dir + '0%d/Sensor/'%train_no\n",
    "    \n",
    "#     old_cols = ['Current_1','Current_2','Current_3', 'Vibration_1']\n",
    "    new_cols = ['Vibration_1','Vibration_2','Vibration_3', 'Current_1']\n",
    "    \n",
    "    def basis_func(idx):\n",
    "#         old_sensor = pd.read_csv(old_input_subdir + '%d.csv'%idx)\n",
    "        new_sensor = pd.read_csv(new_input_subdir + '%d.csv'%idx)\n",
    "        \n",
    "#         old_sensor.columns = old_cols\n",
    "        new_sensor.columns = new_cols\n",
    "        \n",
    "#         if (old_sensor.index.values != new_sensor.index.values).any():\n",
    "#             print('index不一致！')\n",
    "#             return \n",
    "#         # ['Current_1', 'Current_2', 'Current_3'，'Vibration_1']\n",
    "#         sensor = pd.concat([old_sensor[['Current_1', 'Current_2', 'Current_3']], \n",
    "#                             new_sensor[['Vibration_1']]], axis=1)\n",
    "        sensor = new_sensor\n",
    "        if not os.path.exists('./sensors/0%d'%train_no):\n",
    "            os.mkdir('./sensors/0%d'%train_no)\n",
    "        joblib.dump(sensor, './sensors/0%d/%d.lz4'%(train_no, idx), compress='lz4')\n",
    "        \n",
    "    Parallel(n_jobs=48, verbose=1)(delayed(basis_func)(i) for i in range(1,csv_nos+1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=48)]: Using backend LokyBackend with 48 concurrent workers.\n",
      "[Parallel(n_jobs=48)]: Done   2 out of  48 | elapsed:   20.9s remaining:  8.0min\n",
      "[Parallel(n_jobs=48)]: Done  48 out of  48 | elapsed:  1.4min finished\n"
     ]
    }
   ],
   "source": [
    "integrate_sensors(1, 48)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=48)]: Using backend LokyBackend with 48 concurrent workers.\n",
      "[Parallel(n_jobs=48)]: Done   2 out of  48 | elapsed:   40.4s remaining: 15.5min\n",
      "[Parallel(n_jobs=48)]: Done  48 out of  48 | elapsed:  1.5min finished\n"
     ]
    }
   ],
   "source": [
    "integrate_sensors(2, 48)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=48)]: Using backend LokyBackend with 48 concurrent workers.\n",
      "[Parallel(n_jobs=48)]: Done  18 out of  37 | elapsed:  1.1min remaining:  1.2min\n",
      "[Parallel(n_jobs=48)]: Done  37 out of  37 | elapsed:  1.2min finished\n"
     ]
    }
   ],
   "source": [
    "integrate_sensors(3, 37)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import joblib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "joblib.load('./sensors/01/1.lz4')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
